{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.导入import工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants_log</th>\n",
       "      <th>Plasma_glucose_concentration_log</th>\n",
       "      <th>blood_pressure_log</th>\n",
       "      <th>Triceps_skin_fold_thickness_log</th>\n",
       "      <th>serum_insulin_log</th>\n",
       "      <th>BMI_log</th>\n",
       "      <th>Diabetes_pedigree_function_log</th>\n",
       "      <th>Age_log</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.673239</td>\n",
       "      <td>0.802655</td>\n",
       "      <td>0.672552</td>\n",
       "      <td>0.595502</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.465176</td>\n",
       "      <td>0.356534</td>\n",
       "      <td>0.639050</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.239812</td>\n",
       "      <td>0.434206</td>\n",
       "      <td>0.618723</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.286640</td>\n",
       "      <td>0.195523</td>\n",
       "      <td>0.284791</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.760188</td>\n",
       "      <td>0.944101</td>\n",
       "      <td>0.599703</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.186061</td>\n",
       "      <td>0.380165</td>\n",
       "      <td>0.308180</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.239812</td>\n",
       "      <td>0.464683</td>\n",
       "      <td>0.618723</td>\n",
       "      <td>0.434968</td>\n",
       "      <td>0.457607</td>\n",
       "      <td>0.328441</td>\n",
       "      <td>0.068711</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.751240</td>\n",
       "      <td>0.310484</td>\n",
       "      <td>0.595502</td>\n",
       "      <td>0.600411</td>\n",
       "      <td>0.656797</td>\n",
       "      <td>0.965907</td>\n",
       "      <td>0.330870</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.619906</td>\n",
       "      <td>0.640571</td>\n",
       "      <td>0.689516</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.257491</td>\n",
       "      <td>0.093585</td>\n",
       "      <td>0.260660</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.377289</td>\n",
       "      <td>0.447465</td>\n",
       "      <td>0.561052</td>\n",
       "      <td>0.441433</td>\n",
       "      <td>0.403474</td>\n",
       "      <td>0.126835</td>\n",
       "      <td>0.155657</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.829615</td>\n",
       "      <td>0.634817</td>\n",
       "      <td>0.672552</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.503060</td>\n",
       "      <td>0.043865</td>\n",
       "      <td>0.235738</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.380094</td>\n",
       "      <td>0.993262</td>\n",
       "      <td>0.655117</td>\n",
       "      <td>0.692553</td>\n",
       "      <td>0.890236</td>\n",
       "      <td>0.391036</td>\n",
       "      <td>0.062005</td>\n",
       "      <td>0.682494</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.760188</td>\n",
       "      <td>0.690253</td>\n",
       "      <td>0.850956</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.434927</td>\n",
       "      <td>0.115658</td>\n",
       "      <td>0.696441</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants_log  Plasma_glucose_concentration_log  blood_pressure_log  \\\n",
       "0       0.673239                          0.802655            0.672552   \n",
       "1       0.239812                          0.434206            0.618723   \n",
       "2       0.760188                          0.944101            0.599703   \n",
       "3       0.239812                          0.464683            0.618723   \n",
       "4       0.000000                          0.751240            0.310484   \n",
       "5       0.619906                          0.640571            0.689516   \n",
       "6       0.479625                          0.377289            0.447465   \n",
       "7       0.829615                          0.634817            0.672552   \n",
       "8       0.380094                          0.993262            0.655117   \n",
       "9       0.760188                          0.690253            0.850956   \n",
       "\n",
       "   Triceps_skin_fold_thickness_log  serum_insulin_log   BMI_log  \\\n",
       "0                         0.595502           0.527619  0.465176   \n",
       "1                         0.523317           0.527619  0.286640   \n",
       "2                         0.523317           0.527619  0.186061   \n",
       "3                         0.434968           0.457607  0.328441   \n",
       "4                         0.595502           0.600411  0.656797   \n",
       "5                         0.523317           0.527619  0.257491   \n",
       "6                         0.561052           0.441433  0.403474   \n",
       "7                         0.523317           0.527619  0.503060   \n",
       "8                         0.692553           0.890236  0.391036   \n",
       "9                         0.523317           0.527619  0.434927   \n",
       "\n",
       "   Diabetes_pedigree_function_log   Age_log  Target  \n",
       "0                        0.356534  0.639050       1  \n",
       "1                        0.195523  0.284791       0  \n",
       "2                        0.380165  0.308180       1  \n",
       "3                        0.068711  0.000000       0  \n",
       "4                        0.965907  0.330870       1  \n",
       "5                        0.093585  0.260660       0  \n",
       "6                        0.126835  0.155657       1  \n",
       "7                        0.043865  0.235738       0  \n",
       "8                        0.062005  0.682494       1  \n",
       "9                        0.115658  0.696441       1  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "pregnants_log                       768 non-null float64\n",
      "Plasma_glucose_concentration_log    768 non-null float64\n",
      "blood_pressure_log                  768 non-null float64\n",
      "Triceps_skin_fold_thickness_log     768 non-null float64\n",
      "serum_insulin_log                   768 non-null float64\n",
      "BMI_log                             768 non-null float64\n",
      "Diabetes_pedigree_function_log      768 non-null float64\n",
      "Age_log                             768 non-null float64\n",
      "Target                              768 non-null int64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "# 显示基本信息\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "非缺省值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants_log</th>\n",
       "      <th>Plasma_glucose_concentration_log</th>\n",
       "      <th>blood_pressure_log</th>\n",
       "      <th>Triceps_skin_fold_thickness_log</th>\n",
       "      <th>serum_insulin_log</th>\n",
       "      <th>BMI_log</th>\n",
       "      <th>Diabetes_pedigree_function_log</th>\n",
       "      <th>Age_log</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.453441</td>\n",
       "      <td>0.651834</td>\n",
       "      <td>0.666949</td>\n",
       "      <td>0.506745</td>\n",
       "      <td>0.525352</td>\n",
       "      <td>0.422196</td>\n",
       "      <td>0.251366</td>\n",
       "      <td>0.296829</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.266343</td>\n",
       "      <td>0.165941</td>\n",
       "      <td>0.107997</td>\n",
       "      <td>0.124206</td>\n",
       "      <td>0.122691</td>\n",
       "      <td>0.161308</td>\n",
       "      <td>0.171939</td>\n",
       "      <td>0.238177</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.239812</td>\n",
       "      <td>0.540320</td>\n",
       "      <td>0.599703</td>\n",
       "      <td>0.466659</td>\n",
       "      <td>0.520629</td>\n",
       "      <td>0.311985</td>\n",
       "      <td>0.123880</td>\n",
       "      <td>0.097162</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.646277</td>\n",
       "      <td>0.672552</td>\n",
       "      <td>0.523317</td>\n",
       "      <td>0.527619</td>\n",
       "      <td>0.434927</td>\n",
       "      <td>0.209198</td>\n",
       "      <td>0.235738</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.673239</td>\n",
       "      <td>0.766842</td>\n",
       "      <td>0.737819</td>\n",
       "      <td>0.561052</td>\n",
       "      <td>0.532006</td>\n",
       "      <td>0.530852</td>\n",
       "      <td>0.356135</td>\n",
       "      <td>0.491479</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       pregnants_log  Plasma_glucose_concentration_log  blood_pressure_log  \\\n",
       "count     768.000000                        768.000000          768.000000   \n",
       "mean        0.453441                          0.651834            0.666949   \n",
       "std         0.266343                          0.165941            0.107997   \n",
       "min         0.000000                          0.000000            0.000000   \n",
       "25%         0.239812                          0.540320            0.599703   \n",
       "50%         0.479625                          0.646277            0.672552   \n",
       "75%         0.673239                          0.766842            0.737819   \n",
       "max         1.000000                          1.000000            1.000000   \n",
       "\n",
       "       Triceps_skin_fold_thickness_log  serum_insulin_log     BMI_log  \\\n",
       "count                       768.000000         768.000000  768.000000   \n",
       "mean                          0.506745           0.525352    0.422196   \n",
       "std                           0.124206           0.122691    0.161308   \n",
       "min                           0.000000           0.000000    0.000000   \n",
       "25%                           0.466659           0.520629    0.311985   \n",
       "50%                           0.523317           0.527619    0.434927   \n",
       "75%                           0.561052           0.532006    0.530852   \n",
       "max                           1.000000           1.000000    1.000000   \n",
       "\n",
       "       Diabetes_pedigree_function_log     Age_log      Target  \n",
       "count                      768.000000  768.000000  768.000000  \n",
       "mean                         0.251366    0.296829    0.348958  \n",
       "std                          0.171939    0.238177    0.476951  \n",
       "min                          0.000000    0.000000    0.000000  \n",
       "25%                          0.123880    0.097162    0.000000  \n",
       "50%                          0.209198    0.235738    0.000000  \n",
       "75%                          0.356135    0.491479    1.000000  \n",
       "max                          1.000000    1.000000    1.000000  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 显示统计信息\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(dual = False,tol = 0.0001,C = 1,fit_intercept = True,intercept_scaling = 1,class_weight = None,random_state = None,solver = 'lbfgs',max_iter = 10000,multi_class = 'auto',verbose = 0,warm_start = False,n_jobs = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.48550065 0.51107112 0.46557628 0.43770156 0.48899568]\n",
      "cv logloss is: 0.4777690601491524\n"
     ]
    }
   ],
   "source": [
    "#交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#数据集比较大，采用5折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 超参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='auto',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='liblinear',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=4,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "                         'penalty': ['l1', 'l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver = 'liblinear',multi_class = \"auto\")\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,return_train_score=True)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4707096878696069\n",
      "{'C': 10, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过网格搜索可知，最合理的惩罚函数为L2,优化器为liblinear，最优交叉熵损失函数系数应该在10的附近。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores= 0.5723433698020599\n",
      "C= [10.]\n",
      "coef_:\n",
      " [[-1.01359421  3.71520755  0.74559824  2.00905009 -2.02582063  1.86703618\n",
      "   1.70718273  2.79827282]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [ 0.4, 1, 10, 50, 100]\n",
    "lr_cv= LogisticRegressionCV(Cs=Cs,cv=5,penalty='l2',scoring='neg_log_loss',solver='liblinear')\n",
    "lr_cv.fit(X_train,y_train)\n",
    "print('scores=',-lr_cv.scores_[1].mean())\n",
    "print('C=',lr_cv.C_)\n",
    "print('coef_:\\n',lr.coef_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 通过5折逻辑回归校验，发现最优交叉熵损失函数系数的值为10。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUZdrH8e89qfQaQAgQSoIUESTSEZEqKNgBG/rqqruo2FhxcVdFd9W1r+IKVqyABUVwaQoiKEJQQCB0EEKNtIAQ0u73jzNgCAMMyUxOMnN/rmsuZs55zpnfQcyd55TnEVXFGGOMKcjjdgBjjDElkxUIY4wxPlmBMMYY45MVCGOMMT5ZgTDGGONTpNsBAqV69eqakJDgdgxjjClVFi9e/JuqxvlaFzIFIiEhgZSUFLdjGGNMqSIiv55sXVBPMYlIHxFZLSLrRGSEj/UviMgS72uNiOzLt26IiKz1voYEM6cxxpgTBa0HISIRwGigJ5AGLBKRyaq68mgbVb03X/u7gNbe91WBR4BkQIHF3m33BiuvMcaY4wWzB9EWWKeqG1Q1CxgPDDhF+8HAR973vYGZqrrHWxRmAn2CmNUYY0wBwbwGUQfYku9zGtDOV0MRqQ80AL45xbZ1fGx3G3AbQL169Yqe2BhTKmVnZ5OWlkZmZqbbUUqs2NhY4uPjiYqK8nubYBYI8bHsZAM/DQI+UdXcM9lWVccCYwGSk5NtUCljwlRaWhoVKlQgISEBEV8/PsKbqrJ7927S0tJo0KCB39sF8xRTGlA33+d4YNtJ2g7ij9NLZ7qtMSbMZWZmUq1aNSsOJyEiVKtW7Yx7WMEsEIuARBFpICLROEVgcsFGItIEqAL8kG/xdKCXiFQRkSpAL+8yY4zxyYrDqRXm7ydop5hUNUdE7sT5wR4BvKWqK0RkFJCiqkeLxWBgvOYbd1xV94jI4zhFBmCUqu4JRs5DWTm8Nmd9MHbtipqVYhl0fj0iPPY/izGnMnCM8zvphNs7uJyk5Arqg3Kq+hXwVYFl/yjw+dGTbPsW8FbQwnkdzsrl5dnrgv01xSL/1B7XtavvXhBjwlD58uU5ePAgAH369GHBggV07tyZKVOmnNB26NChzJ8/n6ysLDZu3EiTJk0AePjhh7nqqqv8+r5Jkyaxbt06hg8fHriDKCBknqQurGrlY9j4ZD+3YwSEqjJwzAKen7GG/ufWpkKs/3crGGMCZ/jw4Rw6dIgxY8b4XD969GgANm3axCWXXMKSJUt8tsvJySEy0veP6csvvzwwYU/BBusLISLCw5c0ZffvWbwaQqfNjCltunfvToUKFQq1befOnRk5ciQXXHABr7zyCl988QXt2rWjdevW9OrVi127dgHwxhtvcM899wBw/fXXM2zYMDp27EjDhg2ZNGlSQI4j7HsQoaZlfGWuaF2HN+dt5Nq29ahbtazbkYwpVo99uYKV2zJO227ldqfN0WsRp9KsdkUeubR5kbP5KyMjg7lz5wKwd+9e+vfvj4jw2muv8dxzz/H000+fsM2uXbuYP38+v/zyC9dcc01AehjWgwhBD/Rugkfg39NXux3FGFMIgwYNOvZ+8+bN9OrVi3POOYfnn3+eFStW+NzmsssuQ0Ro2bIlW7duDUgO60GEoNqVy3Bbl4b855t13NQxgTb1q7gdyZhi4+9v+iX5LqZy5codez906FD+9re/0bdvX2bNmsVTTz3lc5uYmJhj71UD89yw9SBC1O1dGxFXIYYnpq4M2D8WY0zx279/P3Xq1EFVGTduXLF+txWIEFUuJpLhvZrw8+Z9TFm23e04xoSVLl26cPXVV/P1118THx/P9OmFf8730Ucf5fLLL6dr167UrFkzgClPT0Llt8vk5GS1CYOOl5unXPryPPYfzubr+7sSGxXhdiRjgiI1NZWmTZue0TYl+RRTsPj6exKRxaqa7Ku99SBCWIRHeLhfU7buO8xb8ze6HceYEmXC7R3CqjgUhhWIENexcXV6NK3Jq7PXk37giNtxjDGliBWIMPBQ37PJzM7lhVlr3I5ijClFrECEgUZx5bm+fX3GL9zM6h0H3I5jjCklrEAArJgEmfvdThFUw7onUj4mkn9+lep2FGNMKWEFYvd6+PgmeDoBJt8F2352O1FQVCkXzd3dE5m7Jp05q3e5HccY973dz3mZk7ICUa0R/Gk2tLoOln0MYy+Esd3gp/cg65Db6QLqxg4JJFQryz+nppKTm+d2HGNCSvny5QFYsmQJHTp0oHnz5rRs2ZIJEyac0Hbo0KG0atWKZs2aUaZMGVq1akWrVq345JNP/P6+SZMm8cwzzwQsvy/2HER+h/fBsgmQ8hakr4KYStBqMLS5GWqcHZigLpu+Yge3v7eYxy9rwQ3tbc4IExoK8xzEsd7DzVMDkuHofBBr1qxBREhMTGTbtm20adOG1NRUKleufMI2R4f7Xr58uc99nmq478Kw5yCKokxlaHc7/GUB3PQVJPaERW/Cq+2cf0y/fAI5WW6nLJJezWrSrkFVXpi5hozMbLfjGBNykpKSSExMBKB27drUqFGD9PR0v7e34b5LOhFI6OS8Dj4FS96HlLfh01ugXBy0vh7a3ARVEtxOesZEhL9f0oxLX5nH6NnreOjiM/yty5iS7n8jYMcvp2+3Y5nzpz/XIWqdAxf7HiTvVBYuXEhWVhaNGjU6o+1KynDfViBOp3wcdL4XOg6D9d84p5/mvwTzXoTGPSD5/yCpN3hKzzAWLepU4orW8bw9bxPXt6tvc0YYEwTbt2/nhhtuYNy4cXg8Z3aypuBw39dccw07duzgyJEjJCUl+dzGhvt2k8cDiT2c1/40+OldWDwOxg+GivHQZgicdyNUqOV2Ur8M792Eqb9s46lpqxh97XluxzEmcPz9TT/A1yDyy8jIoF+/fjzxxBO0b9/+jLe34b5Ls0rx0O1vcO9yuOY9qJ4Is/8JLzSHCTfAhjmQV7LvEqpVKZbbL2jE1GXbWfzrHrfjGBMysrKyuPzyy7nxxhu5+uqri7w/G+67tIqIgmb94cbP4a6foP2fYdM8eHcAvJIM378Ch0ruD9/buzakRoUYRk1JJS8vNO5mM8ZtEydOZO7cubzzzjvHbl9dsmRJofdnw30HQIkZ7js7E1Z+ASlvwpYfISIGWlzhXKuIP9+5AF6CfJyyheGfLOOlQa0Y0KqO23GMKZSScJtraXCmt7naNYhAi4qFcwc6rx3LYfHbsHQCLP0Iap4DyTdDy2sgpoLbSQG48rx4xv2wiX9PW03v5rVszggTPsKoMBSWnWIKplotoN9zcH8qXPIiCDD1PnjubJhyr3+34gWZxyOM7NuMrfsO8+Y8mzPCGPMHKxDFIaaC03O4/Tu49WtoNgCWfAivdYY3esKSjyD7sGvxOjSqRq9mNXl19jp2Hch0LYcxRREqp8uDpTB/P1YgipMIxCfDZa/CfanQ+0k4vBc+vwOebwrTR8Jv61yJ9lDfphzJyeOFmTZnhCl9YmNj2b17txWJk1BVdu/eTWxs7BltZxep3aYKm75zhvRYNQXycqBBV+ei9tn9nDulismoL1fyzvcbmXp3F5qeVbHYvteYosrOziYtLY3MTOsBn0xsbCzx8fFERR3/M+VUF6mtQJQkB3bCz94H8PZvgfI1nYfvzhsClesG/ev3Hcqi6zNzOKdOJd67pS1Swu64MsYEng3WV1pUqAkXDIdhS2HwBDirFcx9Fl5qCR8OgrUzIS83aF9fuWw0w7onMm/db8xZ7f/gYsaY0GQ9iJJu32ZY/I4ztMfv6VC5njP8eOsbnHGiAiw7N4/eL8xFBKbdcwFREfY7hDGhzLUehIj0EZHVIrJOREacpM01IrJSRFaIyIf5lueKyBLva3Iwc5ZoletB93/AvSvhqrehcn34+jHnovYn/+c8uR3AIh8V4eGhvk1Zn/474xduDth+jTGlT9B6ECISAawBegJpwCJgsKquzNcmEZgIXKSqe0Wkhqru8q47qKrl/f2+kO1B+JK+xnkAb8kHzlza1Zs4F7XPHeTMaVFEqsq1r//Iqh0ZzBnejUpliu9CuTGmeLnVg2gLrFPVDaqaBYwHBhRo8ydgtKruBThaHMxpxCVBnyfhvlUw4FXnOYtpDzoP4H05DLJ+L9LuRYSR/Zqy73A2o2e7c9utMcZ9wSwQdYAt+T6neZfllwQkich8EVkgIn3yrYsVkRTv8st8fYGI3OZtk3ImMzaFjOiy0Po6+NPXcPtciK3oXK+YOARyizZbXIs6lbjqvHjemb+JX3cXreAYY0qnYBYIX/dIFjyfFQkkAhcCg4E3ROToOZJ63m7PtcCLInLClEyqOlZVk1U1OS4u8BdsS5WzzoUH1sClL8G6mfDF0CIPOf5A7yZEeISnp60KUEhjTGkSzAKRBuS/eT8e2OajzReqmq2qG4HVOAUDVd3m/XMDMAdoHcSsoaPNTXDR32HZBJjxcJEuYNesGMsdXRvx1S87WLSp5A5bbowJjmAWiEVAoog0EJFoYBBQ8G6kz4FuACJSHeeU0wYRqSIiMfmWdwJWYvzT5X5odwcsGA3zXyzSrv50QQNqVYzliSkrbc4IY8JM0AqEquYAdwLTgVRgoqquEJFRItLf22w6sFtEVgKzgeGquhtoCqSIyFLv8qfy3/1kTkPEGeepxVUw61HnGYpCKhsdyV/7NGFp2n4mLy3YATTGhDJ7UC6U5WTBR4Ngw2xnatSmlxRqN3l5yoDR8/nt4BG+uf9CykTbnBHGhAobaiNcRUbDwPeg9nneh+rmF2o3Ho/wcL+mbN+fyZvzNgQ4pDGmpLICEeqiy8F1H0OVBKc3UchJito1rEaf5rV4dc56dmXYiJnGhAMrEOGgbFW44TPngbr3roA9hZs5bsTFZ5Odm8dzM2zOCGPCgRWIcFEpHm6YBHnZ8N7lztDiZyihejmGdEhg4uItrNyWEYSQxpiSxApEOIlrAtd9Agd3wgdXOuM4naG7LkqkUpkonpi60mbvMibEWYEIN/HJzoXrXanw0bWQfWbXEyqVjeKe7ol8v34336yyobOMCWVWIMJR4x5w+Rj4dR58egvk5pzR5te1r0/DuHL886tUsnOLNpyHMabksgIRrs65Cvo87cyDPfXeMxqSIyrCw8i+TdmQ/jsf/mhzRhgTqqxAhLP2d0CXB5wnrb95/Iw2vejsGnRqXI0XZ61h/6GijRxrjCmZrECEu4sehvOGwHfPwYL/+r2ZiDCybzP2Hc7m5W/WBjGgMcYtViDCnQhc8gI0vRSmjYBlE/3etFntilzTpi7jftjEpt9szghjQo0VCAOeCLjiDUjoAp//GdbO8nvT+3slERXh4an/2ZwRxoQaKxDGERULgz6EGs1g4g2wZZFfm9WoGMufuzZi2ood/Lhhd5BDGmOKkxUI84fYinD9p1C+Jnx4Nezyr1dwa5eGnFUpliemptqcEcaEECsQ5njlazhDckREw/tXwL4tp92kTHQED/Y5m1+27ufzJVuLIaQxpjhYgTAnqtoArv8Mjhx0isTvpz911P/c2pwbX4l/T1vN4azcYghpjAk2KxDGt1ot4NrxsG+zc7rpyMFTNvd4hIcvacaOjExe/87mjDAmFFiBMCdXvyNc9TZs+9m5cJ2Tdcrm5ydUpe85tfjvnPXstDkjjCn1rECYUzu7L1z6H1j/DXx+B+SdeuylB/ucTW6e8uz01cUU0BgTLFYgzOmddwP0eBSWfwrTHjzluE31q5Xjpk4JfPJTGsu3nvlw4saYksMKhPFPp3ugw52wcCzMffaUTYd2a0yVstH8c2qqzRlhTClmBcL4RwR6Pg4tB8HsJyDl7ZM2rVQmint7JPLDht3MSrU5I4wpraxAGP95PDDgFUjsBVPvg5VfnLTp4Lb1aFyjPP/6KpWsHJszwpjSyAqEOTMRUXD1OIg/Hz69FTbO9dks0jtnxMbffueDH38t5pDGmECwAmHOXHRZGDweqjZypi3dtsRnswubxNElsTovzlrLvkOnvkXWGFPyWIEwhVO2KtzwGZSpDO9fCbvXn9BERBjZrykHMrN5+Zt1LoQ0xhSFFQhTeBVrO+M2ofDeZZCx/YQmZ9eqyMDz6/LuD5vYaHNGGFOqWIEwRVM9Ea77BA7tcXoSh/ed0OTenklER3h48qtUFwIaYwrLCoQpujrnwcD34bc18NEgyD583OoaFWL5S7fGzFi5kx/W25wRxpQWViBMYDTqBleMhc0L4OObITfnuNW3dG5AncpleGLqSpszwphSwgqECZwWV0DfZ2DN/+DLu48bkiM2KoK/9mnCim0ZfPazzRlhTGlgBcIEVts/QdcRsOQDmPXIcav6n1ubVnUr88z0VRzKyjnJDowxJUVQC4SI9BGR1SKyTkRGnKTNNSKyUkRWiMiH+ZYPEZG13teQYOY0AXbhCEi+Bea/BN+/fGyxiPD3S5qyM+MIY+fanBHGlHSRwdqxiEQAo4GeQBqwSEQmq+rKfG0SgYeATqq6V0RqeJdXBR4BkgEFFnu33RusvCaARJxTTYd2w4yHoWx1aDUYgDb1q9Kv5VmM+XYDg86vR61KsS6HNcacTDB7EG2Bdaq6QVWzgPHAgAJt/gSMPvqDX1WPjuzWG5ipqnu862YCfYKY1QSaJ8K5aN2gK3wxFNZMP7ZqhHfOiGdszghjSrRgFog6QP4Z79O8y/JLApJEZL6ILBCRPmewLSJym4ikiEhKenp6AKObgIiMgUEfQK1zYOIQ5w4noG7VstzcOYFPf0rjlzSbM8KYkiqYBUJ8LCt4f2MkkAhcCAwG3hCRyn5ui6qOVdVkVU2Oi4srYlwTFDEV4PpPoVId+PAa2OmcYRzarTHVykXzxNSVNmeEMSVUMAtEGlA33+d4YJuPNl+oaraqbgRW4xQMf7Y1pUW56nD9ZxBVFt6/Avb+SsXYKO7tmcSPG/cwY+VOtxMaY3wIZoFYBCSKSAMRiQYGAZMLtPkc6AYgItVxTjltAKYDvUSkiohUAXp5l5nSqkp9p0hkH4L3LoeD6Qw6vy6JNcrzpM0ZYUyJFLQCoao5wJ04P9hTgYmqukJERolIf2+z6cBuEVkJzAaGq+puVd0DPI5TZBYBo7zLTGlWsxlcOxEytsIHVxGZ8zsj+zVl0+5DvLfA5owwpqSRUDn/m5ycrCkpKW7HMP5YPQ3GXwsJneG6j7nx3aUs3bKPb4dfSOWy0W6nMyasiMhiVU32tc6epDbFr0kfGDAaNn4Ln/2JkX2SOJCZzUtfr3U7mTEmHysQxh2tBkOvJ2DlFzT56TEGnV+X9374lQ3pB91OZozxOuMCISIeEakYjDAmzHS8CzoNg5S3GFnuC2KjIvjXV6vcTmWM8fKrQIjIhyJSUUTKASuB1SIyPLjRTFjo8Ri0up5yPzzLq0k/MSt1J9+v+42BY35g4Jgf3E5nTFjztwfRTFUzgMuAr4B6wA1BS2XChwhc+hI06UuXtU9zQ4WfeGJqqj08Z0wJ4G+BiBKRKJwC8YWqZuPjyWZjCiUiEq56C6nXnsdyXqLKzvn8djDL7VTGhD1/C8QYYBNQDpgrIvWBjGCFMmEoqgwMHo/EJfJGzAtU3vcLuTbznDGu8qtAqOp/VLWOqvZVx694n4A2JmDKVEZumISnXHVej3ia9tvfdTuRMWHN34vUw7wXqUVE3hSRn4CLgpzNhKMKtYi5eTJR5HI1X7Nrjz1Ab4xb/D3F9H/ei9S9gDjgZuCpoKUy4a1aI9ZENKa+ZxdrJjzsdhpjwpa/BeLo8Nt9gbdVdSm+h+Q2JiDKRCrz8s6h/Y6P2LVmkdtxjAlL/haIxSIyA6dATBeRCoANv2mCZlS1Z3gtbiT7qMCRSXdCXq7bkYwJO/4WiFuAEcD5qnoIiMY5zWRM0GRHV+bbhvdT9/AqfvvmP27HMSbs+HsXUx7OpD0Pi8izQEdVXRbUZMYAXa+4nbnaigrzn4J9m92OY0xY8fcupqeAYTjDbKwE7haRJ4MZzIS3Cbd3YMLtHaheIZZVbR4jN0858Nk9YE9YG1Ns/D3F1BfoqapvqepbQB+gX/BiGfOHgT078YoMpMLmr2HFJLfjGBM2zmQ018r53lcKdBBjTqZSmSjKXzCUZXkNyJ46HA7vdTuSMWHB3wLxJPCziLwjIuOAxcC/ghfLmOPd1LkxT0f9Gc/hPTDzH27HMSYs+HuR+iOgPfCZ99VBVccHM5gx+ZWNjqTHRb14I+di+Old2DTP7UjGhLxTFggROe/oCzgLSAO2ALW9y4wpNte2q8eEstexw1ML/fIeyM50O5IxIS3yNOufO8U6xcZjMsUoJjKC23qcw18/H8K7u5+Gec9Dt7+5HcuYkHXKAqGqNmKrKVGubBPPmLkd+ebIhXT77nmk+RVQ42y3YxkTkvx9DuIKH6/uIlIj2AGNyS8qwsM9PRIZfmAg2RFl4cu7Ic9GfTEmGM5kqI03gOu8r9eB+4D5ImJTj5pidWnL2sTViud5zxDY8iMsftvtSMaEJH8LRB7QVFWvVNUrgWbAEaAd8GCwwhnji8cj3N+rCa/tb8fOau1g1qOQsd3tWMaEHH8LRIKq7sz3eReQpKp7gOzAxzLm1Ho0rUGrulW488CNaG4W/O+vbkcyJuT4WyC+E5EpIjJERIYAk3Hmpi4H7AtePGN8ExGG927CoowqLE74E6ROhlVT3Y5lTEjxt0AMBd4GWgGtgXHAUFX93e50Mm7p1Lg6HRtVY+jGTuTFNYWpD0BmhtuxjAkZ/j5JrcA84BtgFjDXu8wYVz3Quwk7f8/js/gH4cB2+OZxtyMZEzL8vc31GmAhcBVwDfCjiFwVzGDG+OO8elXo0bQGo34uy5E2t8LC12GLTVFqTCD4e4ppJM5sckNU9UagLfD34MUyxn/39WxCRmYOr0VcCxVrO89G5Nq9E8YUlb8FwqOqu/J93u3PtiLSR0RWi8g6ERnhY/1NIpIuIku8r1vzrcvNt3yynzlNGGpWuyKXtDyLMQt2kXHRk7BrJXxvU5QaU1T+FohpIjLd+wP9JmAq8NWpNhCRCGA0cDHOcxODRaSZj6YTVLWV9/VGvuWH8y3v72dOE6bu65nEkZw8XtySCE37w5ynYfd6t2MZU6r5e5F6ODAWaAmcC4xV1dM9INcWWKeqG1Q1CxgPDChKWGNOpmFcea48rw7v//grOzqNgshYmGJTlBpTFH7PKKeqn6rqfap6r6r6M+9jHZyhwY9K8y4r6EoRWSYin4hI3XzLY0UkRUQWiMhlvr5ARG7ztklJT0/391BMiLq7eyIovLTwAPR8FDbOhSUfuh3LmFLrdPNBHBCRDB+vAyJyuhvOxceygr/OfYnzlHZLnNtnx+VbV09Vk4FrgRdFpNEJO1Mdq6rJqpocFxd3mjgm1MVXKcu17eoxMSWNTfWvhrrtYcZIOGi/PBhTGKcsEKpaQVUr+nhVUNWKp9l3GpC/RxAPbCuw/92qesT78XWgTb5127x/bgDm4DygZ8wp/aVbI6IjPLzw9Tq49CU4chCm25wRxhSG36eYCmERkCgiDUQkGhiEM0THMSJyVr6P/YFU7/IqIhLjfV8d6ASsDGJWEyJqVIjlpk4JTF66jVV5taHLffDLRFg3y+1oxpQ6QSsQqpoD3AlMx/nBP1FVV4jIKBE5elfS3SKyQkSWAncDN3mXNwVSvMtnA0+pqhUI45fbL2hI+ZhInpuxBjrfB9USYcq9kPW729GMKVUkVEbMSE5O1pSUFLdjmBLi5a/X8tzMNXw+tBOtclfAO32h413Q6wm3oxlToojIYu/13hME8xSTMa65uXMDqpaL5tnpqyGhE5w3BH4YDduWuB3NmFLDCoQJSeVjIvnLhY2Yt+43fli/G3o+BmWrw5fDIDfH7XjGlApWIEzIur59fWpVjOXZGavR2Mpw8dOwfQksHON2NGNKBSsQJmTFRkVwd/dEFv+6l9mrd0HzyyGxN3zzBOz91e14xpR4ViBMSLs6OZ761cry7PQ15CnQ7zlAYOp9NgyHMadhBcKEtKgID/f2SGLl9gy+Wr4dKteF7n93notY/qnb8Ywp0axAmJB36bm1SapZnudnriEnNw/a3ga1W8O0EXBoj9vxjCmxrECYkBfhEe7v1YQN6b/z2c9bwRMBl/7HKQ4z/+F2PGNKLCsQJiz0alaTc+Mr8dKstRzJyYWzWkLHO+Hn92Djd27HM6ZEsgJhwoKI04vYuu8w4xd6R6HvOgKqJDjPRmRnuprPmJLICoQJG10Sq9OuQVVemb2Ow1m5EF0WLnkB9qyH7551O54xJY4VCBM2RIThvZuQfuAI437Y5CxsdBG0HATzXoCdNh6kMflZgTBhJTmhKt2axPHfOevJyMx2Fvb+J8RUdE415eW5G9CYEsQKhAk79/dqwv7D2bzx3UZnQbnq0PtfkLYQFr/lbjhjShArECbstKhTiX7nnMWb321g90HvhIbnDoKGF8LMRyFj2ym2NiZ8WIEwYenenkkczs7ltW/XOwtEnAvWednw1XB3wxlTQliBMGGpcY3yXHFePON++JUd+723uFZtCBeOgFVTIPVLdwMaUwJYgTBha1j3RFSVl79Z+8fCDndCzRZOLyIzw71wxpQAViBM2KpbtSyDzq/HhEVb2Lz7kLMwIsoZhuPADvh6lLsBjXGZFQgT1u66qDGREcKLs9b8sTC+DbS7HRa9AVsWuhfOGJdZgTBhrUbFWIZ0SGDSkq2s2XngjxUXPQwV68DkuyEny72AxrjICoQJe3d0bUS56Eien5GvFxFTAfo9C+mp8P1L7oUzxkVWIEzYq1Iumlu7NGDaih0sS9v3x4omF0Ozy+DbZ+C3de4FNMYlViCMAW7p3IAqZaN4Nn8vAuDipyEyFqbcY1OUmrBjBcIYoEJsFH++sBFz16Tz44bd+VbUgp6Pwabv4Of33QtojAusQBjjdWOHBGpWjOHZGavR/L2F84ZAvY4w42E4uMu9gMYUMysQxnjFRkVw50WJLNq0l2/XpP+xwuOBS1+E7EMw7SH3AhpTzKxAGJPPwOS61K1a5sReRFwT6HI/LP8E1s50L6AxxcgKhDH5REd6uKd7Esu3ZjBt+Y7jV3a+F6onwZT7IOt3dwIaU4ysQBhTwGWt69C4RnjvAKgAABENSURBVHmem7mG3Lx8vYjIGGcYjv2bYfa/3AtoTDGxAmFMAREe4f6eSazbdZDPf956/Mr6HaDNzbDgVdj2szsBjSkmViCM8aFPi1q0qFORF2atISunwDSkPR6FcnHOMBy5OW7EM6ZYBLVAiEgfEVktIutEZISP9TeJSLqILPG+bs23boiIrPW+hgQzpzEFiQgP9GpC2t7DTEjZcvzKMpXh4n/DjmXw43/dCWhMMQhagRCRCGA0cDHQDBgsIs18NJ2gqq28rze821YFHgHaAW2BR0SkSrCyGuNL16Q4zk+owstfryUzO/f4lc0GQNLFzrWIvZtcyWdMsAWzB9EWWKeqG1Q1CxgPDPBz297ATFXdo6p7gZlAnyDlNMYnEWF477PZdeAI7/6wqeBKZzA/8cDU+20YDhOSglkg6gD5++Zp3mUFXSkiy0TkExGpeybbishtIpIiIinp6ekFVxtTZG0bVOWCpDj+O2c9BzKzj19ZKR66/wPWzYJfPnEnoDFBFMwCIT6WFfw160sgQVVbArOAcWewLao6VlWTVTU5Li6uSGGNOZnhvZqw91A2b87beOLK82+FOm1g2gg4tKf4wxkTRMEsEGlA3Xyf44Ft+Ruo6m5VPeL9+DrQxt9tjSku58RXok/zWrzx3Ub2/l5g8iBPhPNsROY+mPF3dwIaEyTBLBCLgEQRaSAi0cAgYHL+BiJyVr6P/YFU7/vpQC8RqeK9ON3Lu8wYV9zXK4nfs3J47dv1J66s1QI63gVL3ocN3xZ/OGOCJGgFQlVzgDtxfrCnAhNVdYWIjBKR/t5md4vIChFZCtwN3OTddg/wOE6RWQSM8i4zxhVJNStweas6jPthEzszMk9s0PVBqNLAmTci+3Cx5zMmGERD5O6L5ORkTUlJcTuGCWGbdx/ioufmMLhtPR6/rMWJDTbMgXcHOIP6df9HseczpjBEZLGqJvtaZ09SG+OnetXKMvD8uoxftJktew6d2KDhhXDutTD/Jdi5orjjGRNwViCMOQN3XZSIR4SXvl7ru0GvJyC2kjMMR16u7zbGlBJWIIw5A7UqxXJjh/p89lMa63YdOLFBuWrQ+0nYmgIpbznL3u7nvIwpZaxAGHOG/nxhY8pERfDCzJP0IlpeAw27wazHYP9W322MKQWsQBhzhqqWi+aWLg2Z+st2lm/df2IDEbjkBcjLgf/9tfgDGhMgViCMKYRbuzSgUpkonpux2neDqg2g20Owagoc+q14wxkTIFYgjCmEirFR3NG1EbNXp5Oy6SSP6LQfCrXOgT0bnN6EMaWMFQhjCmlIx/rEVYjh39NX4/N5oohIuPQlyM2C7Uvh+1fg4K7iD2pMIVmBMKaQykZHcme3xizcuIfv1p7kNFKdNlD9bPBEwoyR8HxT+GgwpE6B3Gzf2xhTQliBMKYIBrWtS53KZXh2xkl6EQDlqsNZ58JffoT2f4Gti2HCdfDc2TDtb7BjefGGNsZPViCMKYKYyAiG9UhkWdp+pq/YeerGNc6GXo/DvSth8ASo3xEWjoXXOsGYC+DHsTZkuClRrEAYU0RXtK5Dw7hyPD9zNbl5foxtFhEJTfrAwPfg/tXQ52lnRrr/DYfnmsDEG2HNDMi1C9vGXVYgjCmiyAgP9/VMYs3Og3y59AynLSlXDdrfAXd8B3fMg+RbYNM8+PBqeKE5zHwE0tcEJ7gxp2GjuRoTAHl5yiUvz+PgkRy+vr8rURFF+N0rJwvWTIMlH8LaGaC5EH8+tLoOWlzhjPVkTIDYaK7GBJnHIzzQO4nNew7xcUpa0XYWGQ3N+sO14+G+VOj5OBw54Mw18WwSfHorrJ8NeXmBCW/MSVgPwpgAUVWu/O/3bNuXyZzhFxIbFRHIncO2n+DnD2D5J5C5HyrVhXMHQ6trnSe3jSkE60EYUwxEhOG9z2ZHRibvL/g10Dt3nqm45Hm4fw1c+SZUT4K5z8B/WsHbfZ3iceRgYL/XhDXrQRgTYNe/8SMrt2cw96/dKB8TycAxPwAw4fYOgf+y/Vth6UfO9Yo96yGqHDS/3OlV1O/oFBZjTsF6EMYUowd6N2HP71m8PW9j8L+sUh244AG4azH833TnIvbKz+GdvvCf1vDtM7BvS/BzmJBkBcKYAGtVtzI9m9Vk7NwN7DuUVTxfKgL12sOAV+CBNXDZa1ApHmY/AS+e48yVvexjyD5cPHlMSLACYUwQ3N8riYNZOYyZu6H4vzy6HLQaDDdNgWFLoeuDsHsDfHYrPNsEvrwHtixyLnyfoRX/6syKf3UOQujiFSrHAcE9FisQxgTB2bUq0v/c2rw9fyNZOS7ejlolwZmXYthSGPIlNLkYlo6HN3vA6HYw70U4sMO9fKZEswJhTJDc2yOJ7Fxl2/4ScFrH44EGF8AVY5xTUJf+B8pUhlmPOCPMfnA1rPgcco64ndSUIFYgjAmShOrluCY5nl0ZRziclevfOE3FIbYitBkCt8yAOxdDp3ucEWU/HuKMBfXVX535K0zYi3Q7gDGh7K6LEhm/aAvLtu6n0d++onxMJOVjIqkQe/QVdezPigWWOe2c9xWPtYsksijDeBRUvTH0eAQuehg2zHaepVj8DiwcAzXPcW6XbXmNM2S5CTtWIIwJotqVy9DsrIocyMzh8tZ1OJCZw4HMbOfPI9nsPZTF5j2Hji0/4sf1ijJRET4KTCQVYqJOXOYtPOULLI+JLPCUtycCGvdwXof3wi+fwJIPYPpDMPPvkNTHGQtK1Z6tCCNWIIwJsqO9hnt7Jp227ZGcXA5m5ngLhlM0MvIVlYNH8hWYzBwyvO+37Tt8bNnh7NzTfk90pMfbYzm+wJQ/Vli6UrFpd+IbbyRp+5fEb5pM9KopJBJJFtEcebmDU1QkAvVEgCcSlUjweLx/Hl3nfe+JROWPP8XjXXe0jXi8bZ33eCKPbXt0O+d1dP9/fIfT1rtP73erJxI59v1/7F8inO88mBuFAr9tLYZnVSCoRfVgbhRCcE5fWoEwpgSJiYwgpnwE1crHFHof2bl5/H7k+AJyXM/F+2dGgYKTfuDgsbYHj+Sfi6InkXSjq2cpfTyLKCuZRO7KI4JcIsgjgiwiOUyE5BFBHpHketcpEeR6PzvrIiSXSPLwHGuXRxS5f3yW4rlOc+yE2eutiuX7gqk6kJpXLyj7tgJhTIiJivBQuWw0lctGF3ofuXl6rHgcPHK0wLQndUIaihDf/x8AqI/fXAs+XnHC5xPa51uieaC5iObh0VxEcxDNQ/JynM95uQi5cPTzsfV/vPdoLujR9XnO8rxcRJ2XR3PZv2wKglKpRa9C/x35K9g1b/+K6YhHaBqEfVuBMMacIMIjVCoTRaUyUcctrxm1EoDmreu4EStgVqwaD0Dzq+53OUnRrVg7KWj7tttcjTHG+BTUAiEifURktYisE5ERp2h3lYioiCR7PyeIyGERWeJ9vRbMnMYYY04UtFNMIhIBjAZ6AmnAIhGZrKorC7SrANwN/FhgF+tVtfRfQTJhLyjDfBtTDILZg2gLrFPVDaqaBYwHBvho9zjwbyAziFmMMcacoWAWiDpA/oHo07zLjhGR1kBdVZ3iY/sGIvKziHwrIl18fYGI3CYiKSKSkp6eHrDgxhjfRlV7hlHVnnE7hikmwbyLydeTIcdu+BIRD/ACcJOPdtuBeqq6W0TaAJ+LSHNVzThuZ6pjgbHgzCgXqODGGN9C5XRZ87/NcztCwATzWILZg0gD6ub7HA9sy/e5AtACmCMim4D2wGQRSVbVI6q6G0BVFwPrgdM/hmqMMSZgglkgFgGJItJARKKBQcDkoytVdb+qVlfVBFVNABYA/VU1RUTivBe5EZGGQCLgwswrxhgTvoJ2iklVc0TkTmA6EAG8paorRGQUkKKqk0+x+QXAKBHJAXKBO1R1T7CyGmOMOZFoIaYdLImSk5M1JSXF7RjGGFOqiMhiVU32tc6epDbGGOOTFQhjjDE+WYEwxhjjkxUIY4wxPlmBMMYY41PI3MUkIunAr0XYRXXgtwDFcVOoHAfYsZRUoXIsoXIcULRjqa+qcb5WhEyBKCoRSTnZrV6lSagcB9ixlFShciyhchwQvGOxU0zGGGN8sgJhjDHGJysQfxjrdoAACZXjADuWkipUjiVUjgOCdCx2DcIYY4xP1oMwxhjjkxUIY4wxPlmB8BKRx0VkmYgsEZEZIlLb7UyFJSLPiMgq7/FMEpHKbmcqLBG5WkRWiEieiJS6WxJFpI+IrBaRdSIywu08RSEib4nILhFZ7naWohCRuiIyW0RSvf+2hrmdqbBEJFZEForIUu+xPBbQ/ds1CIeIVDw6pamI3A00U9U7XI5VKCLSC/jGOyfH0wCq+qDLsQpFRJoCecAY4AFVLTVjunsnvVoD9MSZYXERMFhVV7oarJBE5ALgIPCuqrZwO09hichZwFmq+pOIVAAWA5eVxv8uIiJAOVU9KCJRwDxgmKouCMT+rQfhVWC+63Lkmz+7tFHVGaqa4/24AGe611JJVVNVdbXbOQqpLbBOVTeoahYwHhjgcqZCU9W5QKmfuEtVt6vqT973B4BUoI67qQpHHQe9H6O8r4D97LICkY+I/FNEtgDXAf9wO0+A/B/wP7dDhKk6wJZ8n9MopT+IQpWIJACtgR/dTVJ4IhIhIkuAXcBMVQ3YsYRVgRCRWSKy3MdrAICqjlTVusAHwJ3upj210x2Lt81IIAfneEosf46llBIfy0ptzzTUiEh54FPgngJnEEoVVc1V1VY4ZwraikjATv8FbU7qkkhVe/jZ9ENgKvBIEOMUyemORUSGAJcA3bWEX2g6g/8upU0aUDff53hgm0tZTD7e8/WfAh+o6mdu5wkEVd0nInOAPkBAbiQIqx7EqYhIYr6P/YFVbmUpKhHpAzwI9FfVQ27nCWOLgEQRaSAi0cAgYLLLmcKe98Lum0Cqqj7vdp6iEJG4o3cpikgZoAcB/NlldzF5icinQBOcO2Z+Be5Q1a3upiocEVkHxAC7vYsWlOI7si4HXgbigH3AElXt7W4q/4lIX+BFIAJ4S1X/6XKkQhORj4ALcYaW3gk8oqpvuhqqEESkM/Ad8AvO/+8Af1PVr9xLVTgi0hIYh/PvywNMVNVRAdu/FQhjjDG+2CkmY4wxPlmBMMYY45MVCGOMMT5ZgTDGGOOTFQhjjDE+WYEw5gyIyMHTtzrl9p+ISEPv+/IiMkZE1ntH4pwrIu1EJNr7PqweZDUljxUIY4qJiDQHIlR1g3fRGziD3yWqanPgJqC6d2C/r4GBrgQ1xssKhDGFII5nvGNG/SIiA73LPSLyqrdHMEVEvhKRq7ybXQd84W3XCGgHPKyqeQDeUV+nett+7m1vjGusC2tM4VwBtALOxXmyeJGIzAU6AQnAOUANnKGk3/Ju0wn4yPu+Oc5T4bkn2f9y4PygJDfGT9aDMKZwOgMfeUfS3Al8i/MDvTPwsarmqeoOYHa+bc4C0v3ZubdwZHkntDHGFVYgjCkcX0N5n2o5wGEg1vt+BXCuiJzq/8EYILMQ2YwJCCsQxhTOXGCgd7KWOOACYCHOlI9Xeq9F1MQZ3O6oVKAxgKquB1KAx7yjiyIiiUfnwBCRakC6qmYX1wEZU5AVCGMKZxKwDFgKfAP81XtK6VOceSCW48yj/SOw37vNVI4vGLcCtYB1IvIL8Dp/zBfRDSh1o4ua0GKjuRoTYCJS3juJfDWcXkUnVd3hHa9/tvfzyS5OH93HZ8BDpXg+bhMC7C4mYwJvincSl2jgcW/PAlU9LCKP4MxLvflkG3snF/rcioNxm/UgjDHG+GTXIIwxxvhkBcIYY4xPViCMMcb4ZAXCGGOMT1YgjDHG+PT/tbXbaEVmXEkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "#     plt.plot(log10(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "#     plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当C=10,即log(C)=1时，L2正则化的效果最好。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000], class_weight=None,\n",
       "                     cv=5, dual=False, fit_intercept=True,\n",
       "                     intercept_scaling=1.0, l1_ratios=None, max_iter=100,\n",
       "                     multi_class='ovr', n_jobs=None, penalty='l2',\n",
       "                     random_state=None, refit=True, scoring='neg_log_loss',\n",
       "                     solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]\n",
    "#nCs = 9  #Cs values are chosen in a logarithmic scale between 1e-4 and 1e4.\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.67579821, -0.65028636, -0.57994567, -0.49047738, -0.48287833,\n",
       "         -0.4861858 , -0.48668222],\n",
       "        [-0.67577845, -0.65145008, -0.59091576, -0.51852331, -0.51976861,\n",
       "         -0.52521666, -0.52596832],\n",
       "        [-0.67632053, -0.65261619, -0.58945687, -0.48699887, -0.45098363,\n",
       "         -0.44674949, -0.44634525],\n",
       "        [-0.67558734, -0.65009116, -0.57878034, -0.45877684, -0.41876882,\n",
       "         -0.4145485 , -0.41416005],\n",
       "        [-0.67485172, -0.64825283, -0.58275564, -0.49504063, -0.4808778 ,\n",
       "         -0.48266501, -0.48298612]])}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAASm0lEQVR4nO3df7BfdX3n8eeLRLFFEYWgQIixJe42rC1tv8J0YB2qguhUwypb0M5uWu0wnSlTu5224rK7FNQpbn/YsbUdU6GTOhW0UmsqtsjPUncUcoNYCEiJtC5XEGKD2IxtEfPeP74n7vX2G/K9n3u/99yb+3zM3Pme8zmf7znvM0nuK+d8zo9UFZIkzdVhfRcgSVqeDBBJUhMDRJLUxACRJDUxQCRJTVb3XcBiOuaYY2r9+vV9lyFJy8qOHTu+VlVrZrevqABZv349U1NTfZchSctKki+PavcUliSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpSa8BkuScJPcn2ZXk4hHLD0/ykW757UnWz1q+LsneJL+8WDVLkoZ6C5Akq4D3A68BNgJvSrJxVre3Ao9X1UnAe4H3zFr+XuAvJ12rJOnf6vMI5FRgV1U9WFVPAtcAm2b12QRs7aY/BrwySQCSnAs8COxcpHolSTP0GSAnAA/NmJ/u2kb2qaqngCeAo5McAbwduOxgG0lyYZKpJFO7d+9ekMIlSf0GSEa01Zh9LgPeW1V7D7aRqtpSVYOqGqxZs6ahTEnSKKt73PY0cOKM+bXAwwfoM51kNfBcYA9wGnBekv8NHAXsS/IvVfV7ky9bkgT9Bsh2YEOSFwNfAS4A3jyrzzZgM/BZ4Dzg5qoq4D/u75Dk14C9hockLa7eAqSqnkpyEXA9sAq4qqp2JrkcmKqqbcCVwIeS7GJ45HFBX/VKkr5bhv+hXxkGg0FNTU31XYYkLStJdlTVYHa7d6JLkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCa9BkiSc5Lcn2RXkotHLD88yUe65bcnWd+1n5VkR5K7u89XLHbtkrTS9RYgSVYB7wdeA2wE3pRk46xubwUer6qTgPcC7+navwa8rqpeCmwGPrQ4VUuS9uvzCORUYFdVPVhVTwLXAJtm9dkEbO2mPwa8Mkmq6vNV9XDXvhN4VpLDF6VqSRLQb4CcADw0Y366axvZp6qeAp4Ajp7V543A56vqXydUpyRphNU9bjsj2moufZKczPC01tkH3EhyIXAhwLp16+ZepSRppD6PQKaBE2fMrwUePlCfJKuB5wJ7uvm1wMeB/1pVXzrQRqpqS1UNqmqwZs2aBSxfkla2PgNkO7AhyYuTPBO4ANg2q882hoPkAOcBN1dVJTkKuA54R1X9n0WrWJL0Hb0FSDemcRFwPXAf8NGq2pnk8iSv77pdCRydZBfwS8D+S30vAk4C/meSu7qfYxd5FyRpRUvV7GGHQ9dgMKipqam+y5CkZSXJjqoazG73TnRJUhMDRJLUxACRJDUxQCRJTQwQSVITA0SS1MQAkSQ1MUAkSU0MEElSEwNEktTEAJEkNTFAJElNDBBJUhMDRJLUxACRJDUxQCRJTQwQSVITA0SS1MQAkSQ1mXOAJDksyZGTKEaStHyMFSBJPpzkyCRHAPcC9yf5lcmWJklaysY9AtlYVd8AzgU+BawD/svEqpIkLXnjBsgzkjyDYYB8oqq+BdTkypIkLXXjBsgHgH8AjgBuS/Ii4BuTKkqStPStHqdTVb0PeN+Mpi8n+fHJlCRJWg7GHUR/WzeIniRXJrkTeMWEa5MkLWHjnsJ6SzeIfjawBvgZ4IqJVSVJWvLGDZB0n68F/qiqvjCjTZK0Ao0bIDuSfJphgFyf5DnAvsmVJUla6sYaRAfeCpwCPFhV30xyNMPTWJKkFWrcq7D2JVkLvDkJwF9X1V9MtDJJ0pI27lVYVwBvY/gYk3uBX0jy6/PdeJJzktyfZFeSi0csPzzJR7rltydZP2PZO7r2+5O8er61SJLmZtxTWK8FTqmqfQBJtgKfB97RuuEkq4D3A2cB08D2JNuq6t4Z3d4KPF5VJyW5AHgPcH6SjcAFwMnA8cCNSV5SVd9urUeSNDfjBgjAUcCebvq5C7DtU4FdVfUgQJJrgE0Mj3D22wT8Wjf9MeD3MjyHtgm4pqr+Ffj7JLu69X12Aer6Ny77i53c+7A33ktanjYefySXvu7kBV/vuAHy68Dnk9zC8PLdlzOPo4/OCcBDM+angdMO1KeqnkryBHB01/65Wd89YdRGklwIXAiwbt26eZYsSdpv3EH0q5PcCryMYYC8vaq+Os9tj7qPZPYDGg/UZ5zvDhurtgBbAAaDQdMDICeR3JK03D1tgCT5kVlN093n8UmOr6o757HtaeDEGfNrgYcP0Gc6yWqGp872jPldSdIEHewI5LeeZlkxv+dhbQc2JHkx8BWGg+JvntVnG7CZ4djGecDNVVVJtgEfTvLbDAfRNwB3zKMWSdIcPW2AVNXEnrjbjWlcBFwPrAKuqqqdSS4HpqpqG3Al8KFukHwPw5Ch6/dRhgPuTwE/7xVYkrS4UnXwYYEkbxjR/ARwd1U9tuBVTchgMKipqam+y5CkZSXJjqoazG6fy6NMfgy4pZs/k+FVUC9JcnlVfWhBqpQkLRvjBsg+4Aeq6lGAJC8A/oDhZbe3AQaIJK0w4z6Nd/3+8Og8BrykqvYA31r4siRJS924RyB/k+STwJ928+cxfDf6EcDXJ1KZJGlJGzdAfh54A3AGw5v4tgLX1nAE3nejS9IKNO6d6JXkM8CTDO//uKPGuXxLknTIGvdx7j/J8Ea984CfBG5Pct4kC5MkLW3jnsK6BHjZ/ns+kqwBbmT4hFxJ0go07lVYh826YfAf5/BdSdIhaNwjkL9Kcj1wdTd/PvCpyZQkSVoOxh1E/5UkbwROZ3gV1paq+vhEK5MkLWljv5Gwqq4Frp1gLZKkZeRg7wP5J0a/qCkMr+49ciJVSZKWvIM9zv05i1WIJGl58UoqSVITA0SS1MQAkSQ1MUAkSU0MEElSEwNEktTEAJEkNTFAJElNDBBJUhMDRJLUxACRJDUxQCRJTQwQSVITA0SS1MQAkSQ1MUAkSU16CZAkz09yQ5IHus/nHaDf5q7PA0k2d23fm+S6JF9MsjPJFYtbvSQJ+jsCuRi4qao2ADd1898lyfOBS4HTgFOBS2cEzW9W1b8Hfhg4PclrFqdsSdJ+fQXIJmBrN70VOHdEn1cDN1TVnqp6HLgBOKeqvllVtwBU1ZPAncDaRahZkjRDXwHygqp6BKD7PHZEnxOAh2bMT3dt35HkKOB1DI9iJEmLaPWkVpzkRuCFIxZdMu4qRrTVjPWvBq4G3ldVDz5NHRcCFwKsW7duzE1Lkg5mYgFSVa860LIkjyY5rqoeSXIc8NiIbtPAmTPm1wK3zpjfAjxQVb9zkDq2dH0ZDAb1dH0lSePr6xTWNmBzN70Z+MSIPtcDZyd5Xjd4fnbXRpJ3Ac8FfnERapUkjdBXgFwBnJXkAeCsbp4kgyQfBKiqPcA7ge3dz+VVtSfJWoanwTYCdya5K8nP9rETkrSSpWrlnNUZDAY1NTXVdxmStKwk2VFVg9nt3okuSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJr0ESJLnJ7khyQPd5/MO0G9z1+eBJJtHLN+W5J7JVyxJmq2vI5CLgZuqagNwUzf/XZI8H7gUOA04Fbh0ZtAkeQOwd3HKlSTN1leAbAK2dtNbgXNH9Hk1cENV7amqx4EbgHMAkjwb+CXgXYtQqyRphL4C5AVV9QhA93nsiD4nAA/NmJ/u2gDeCfwW8M2DbSjJhUmmkkzt3r17flVLkr5j9aRWnORG4IUjFl0y7ipGtFWSU4CTquq/JVl/sJVU1RZgC8BgMKgxty1JOoiJBUhVvepAy5I8muS4qnokyXHAYyO6TQNnzphfC9wK/Bjwo0n+gWH9xya5tarORJK0aPo6hbUN2H9V1WbgEyP6XA+cneR53eD52cD1VfUHVXV8Va0HzgD+zvCQpMXXV4BcAZyV5AHgrG6eJIMkHwSoqj0Mxzq2dz+Xd22SpCUgVStnWGAwGNTU1FTfZUjSspJkR1UNZrd7J7okqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmBogkqYkBIklqYoBIkpoYIJKkJgaIJKmJASJJamKASJKaGCCSpCYGiCSpiQEiSWpigEiSmhggkqQmqaq+a1g0SXYDX278+jHA1xawnD4dKvtyqOwHuC9L1aGyL/PdjxdV1ZrZjSsqQOYjyVRVDfquYyEcKvtyqOwHuC9L1aGyL5PaD09hSZKaGCCSpCYGyPi29F3AAjpU9uVQ2Q9wX5aqQ2VfJrIfjoFIkpp4BCJJamKASJKaGCBzkOSdSf42yV1JPp3k+L5rapHkN5J8sduXjyc5qu+aWiX5z0l2JtmXZFlebpnknCT3J9mV5OK+62mV5KokjyW5p+9a5iPJiUluSXJf93frbX3X1CrJs5LckeQL3b5ctqDrdwxkfEmOrKpvdNO/AGysqp/ruaw5S3I2cHNVPZXkPQBV9faey2qS5AeAfcAHgF+uqqmeS5qTJKuAvwPOAqaB7cCbqureXgtrkOTlwF7gj6vqP/RdT6skxwHHVdWdSZ4D7ADOXaZ/JgGOqKq9SZ4BfAZ4W1V9biHW7xHIHOwPj84RwLJM36r6dFU91c1+DljbZz3zUVX3VdX9fdcxD6cCu6rqwap6ErgG2NRzTU2q6jZgT991zFdVPVJVd3bT/wTcB5zQb1VtamhvN/uM7mfBfm8ZIHOU5N1JHgJ+CvhffdezAN4C/GXfRaxgJwAPzZifZpn+sjoUJVkP/DBwe7+VtEuyKsldwGPADVW1YPtigMyS5MYk94z42QRQVZdU1YnAnwAX9VvtgR1sP7o+lwBPMdyXJWucfVnGMqJtWR7ZHmqSPBu4FvjFWWcflpWq+nZVncLwTMOpSRbs9OLqhVrRoaKqXjVm1w8D1wGXTrCcZgfbjySbgZ8AXllLfCBsDn8my9E0cOKM+bXAwz3Vok43XnAt8CdV9Wd917MQqurrSW4FzgEW5EIHj0DmIMmGGbOvB77YVy3zkeQc4O3A66vqm33Xs8JtBzYkeXGSZwIXANt6rmlF6waerwTuq6rf7rue+UiyZv9Vlkm+B3gVC/h7y6uw5iDJtcC/Y3jVz5eBn6uqr/Rb1dwl2QUcDvxj1/S55Xg1GUCS/wT8LrAG+DpwV1W9ut+q5ibJa4HfAVYBV1XVu3suqUmSq4EzGT46/FHg0qq6steiGiQ5A/gb4G6G/9YB/ntVfaq/qtok+UFgK8O/W4cBH62qyxds/QaIJKmFp7AkSU0MEElSEwNEktTEAJEkNTFAJElNDBBpASXZe/BeT/v9jyX5vm762Uk+kORL3ZNUb0tyWpJndtPeCKxeGSDSEpHkZGBVVT3YNX2Q4cMJN1TVycBPA8d0D128CTi/l0KljgEiTUCGfqN7ZtfdSc7v2g9L8vvdEcUnk3wqyXnd134K+ETX7/uB04D/UVX7ALon9l7X9f3zrr/UGw+Bpcl4A3AK8EMM78zenuQ24HRgPfBS4FiGjwq/qvvO6cDV3fTJDO+q//YB1n8P8LKJVC6NySMQaTLOAK7unoT6KPDXDH/hnwH8aVXtq6qvArfM+M5xwO5xVt4Fy5PdC4+kXhgg0mSMekz707UD/DPwrG56J/BDSZ7u3+jhwL801CYtCANEmozbgPO7l/msAV4O3MHwlaJv7MZCXsDw4YP73QecBFBVXwKmgMu6p8OSZMP+d6AkORrYXVXfWqwdkmYzQKTJ+Djwt8AXgJuBX+1OWV3L8B0g9zB8j/vtwBPdd67juwPlZ4EXAruS3A38If//XSE/Diy7p8Pq0OLTeKVFluTZVbW3O4q4Azi9qr7ava/hlm7+QIPn+9fxZ8A7lvn74LXMeRWWtPg+2b3k55nAO7sjE6rqn5NcyvCd6P/3QF/uXjz154aH+uYRiCSpiWMgkqQmBogkqYkBIklqYoBIkpoYIJKkJv8Py8H7J+eY/c4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "Cs = [1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]\n",
    "n_Cs = len(Cs)\n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('logloss')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 保存模型\n",
    "import pickle as pk\n",
    "pk.dump(grid.best_estimator_,open(\"Pima_L2_log.pkl\",\"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进行准确率的预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当penalty='l2',C=10,solver='liblinear'时，accuracy_score= 0.7714285714285715 ,log_loss= 4.317347049363836\n"
     ]
    }
   ],
   "source": [
    "#将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "# 随机采样10%的数据构建测试样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, random_state=687, test_size=0.1)\n",
    "\n",
    "lr= LogisticRegression(penalty='l2',C=10,solver='liblinear').fit(X_train, y_train)\n",
    "y_pred=lr.predict(X_test)\n",
    "\n",
    "accuracy_score=accuracy_score(y_test, y_pred)\n",
    "log_loss=log_loss(y_test, y_pred)\n",
    "print(\"当penalty='l2',C=10,solver='liblinear'时，accuracy_score=\",accuracy_score_my,\",log_loss=\",log_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 通过GridSearchCV查找的模型参数C = 10,能够得到比手动调参更高的准确率accuracy_score= 0.7714285714285715。此时的C=10,惩罚函数为L2."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
